#include "../include/DictProducer.h"


DictProducer::DictProducer()
{
    
}

// 英文词典
void DictProducer::buildEnDict(){
    // 打开语料文件
    DIR * dir = opendir("conf/yuliao/en");
    if(!dir){
        perror("opendir failed");
        return;
    }

    struct dirent * dirent;
    while(dirent = readdir(dir)){
        if(dirent->d_name == "." || dirent->d_name == ".."){
            continue;
        }
        _files.push_back(string("conf/yuliao/en/") + string(dirent->d_name));
    }
    closedir(dir);

    //加载停用词
    ifstream ifs("conf/stop_words/stop_words_eng.txt");
    if(!ifs.good()){
        perror("ifs stop_words_eng failed");
        return;
    }

    string stopBuf;
    while(ifs >> stopBuf){
        _stop.insert(stopBuf);
    }

    //打开语料文件
    for(auto & path : _files){
        ifstream ifs(path);

        if(!ifs.good()){
            perror("ifs files path failed");
            return;
        }
        string txt;
        int cnt = 1; // 词频
        
        while(getline(ifs, txt)){
            istringstream iss(txt);
            string word;
            bool isAlpha = true;
            while(iss >> word){
                for(int i=0; i<word.size(); i++){
                    if(isalpha(word[i])){
                        if(!islower(word[i])){
                            word[i] = tolower(word[i]);
                        }
                    } else {
                        isAlpha = false;
                        continue;
                    }
                }
                if(!_stop.count(word) && isAlpha){ // 检查是否为停用词
                    if(!_dict.count(word)){
                        _dict.insert({word, cnt});
                    } else{
                        _dict[word]++;
                    }
                }
                isAlpha = true;
            }
        }
        ifs.close();
    }
}
void DictProducer::buildEnIndex(){
    size_t cnt = 0;
    for(auto word : _dict){
        for(auto it = word.first.begin(); it != word.first.end(); it++){
            _index[*it].insert(cnt);
        }
        cnt++;
    }
}

void DictProducer::storeEnDict(const string &filepath){
    ofstream ofs;
    ofs.open(filepath);
    if (!ofs.good()){
        perror("ofs failed");
        return;
    }

    for(auto elem : _dict){
        ofs << elem.first << " " << elem.second << endl;
    }
    
    ofs.close();
}
void DictProducer::storeEnIndex(const string &filepath){
    ofstream ofs(filepath);
    if(!ofs.good()){
        perror("ofs failed");
        return;
    }

    for(auto word : _index){
        ofs << word.first;
        for(auto it = word.second.begin(); it != word.second.end(); it++){
            ofs << " " << *it;
        }
    }
    ofs.close();
}

// 中文词典
void DictProducer::buildCnDict(){
    // 打开语料文件
    DIR * dir = opendir("conf/yuliao/cn");
    if(!dir){
        perror("opendir chinese failed");
        return;
    }

    struct dirent * dirent;
    while(dirent = readdir(dir)){
        if(string(dirent->d_name) == string(".") || dirent->d_name == string("..")){
            continue;
        }
        _filesZh.push_back(string("conf/yuliao/cn/") + string(dirent->d_name));
    }
    closedir(dir);

    //加载停用词
    ifstream ifs("jiebadict/stop_words.utf8");
    if(!ifs.good()){
        perror("ifs stop_words_zh failed");
        return;
    }

    string stopBuf;
    while(ifs >> stopBuf){
        _stopZh.insert(stopBuf);
    }
    // 遍历语料库，分词，索引，词典
    unique_ptr<SplitTool> cppjieba(new SplitToolCppjieba);
    string sentence;

    //打开语料文件
    for(auto & path : _filesZh){
        cout << "dealing " << path << "\n";
        ifstream ifs(path);

        if(!ifs.good()){
            perror("ifs files path failed");
            return;
        }
        
        while(getline(ifs, sentence)){
            // 中文分词
            vector<string> cutedWords = cppjieba->cut(sentence);
            for(auto word : cutedWords){
                if(word == string(" ")) continue;

                if(!_stopZh.count(word)){ // 检查是否为停用词
                    _dict[word]++;
                }
            }
        }
        ifs.close();
    }
}

void DictProducer::buildCnIndex(){
    unique_ptr<SplitTool> UTF8(new SplitToolUTF8);
    size_t cnt = 0;
    for(auto word : _dictZh){
        vector<string> cuted = UTF8->cut(word.first);
        for(auto it = cuted.begin(); it != cuted.end(); it++){
            _indexZh[*it].insert(cnt);
        }
        cnt++;
    }
}
void DictProducer::storeCnDict(const string &filepath){
    ofstream ofs(filepath);
    if(!ofs.good()){
        perror("open dictCn file failed");
        return;
    }

    for(auto &elem : _dictZh){
        ofs << elem.first << " " << elem.second << endl;
    }
    ofs.close();
}
void DictProducer::storeCnIndex(const string &filepath){
    ofstream ofs(filepath);
    if(!ofs.good()){
        perror("open dictCn file failed");
        return;
    }

    for(auto &elem : _indexZh){
        ofs << elem.first;
        for(auto cnt : elem.second){
            ofs << " " << cnt;
        }
        ofs << "\n";
    }
    ofs.close();
}